import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CrawlerNew {
	public static String html = "";// 存放网页HTML源代码

	static String s[] = { "jpg", "png", "gif", "bmp","css" };
	static List<String> list = Arrays.asList(s);
	public static HashSet<String> forms = new HashSet<String>(list);

	public static void main(String args[]) {
		HashSet<String> urls=extactURLS("http://www.58.com/changecity.aspx");
		Iterator<String> iter=urls.iterator();
		while(iter.hasNext()){
			startDownload(iter.next());
		}
//		startDownload("http://tj.58.com");
	}

	public static void startDownload(String pageURL) {
		File savePath = new File("webPage");
		if (!savePath.exists())
			savePath.mkdir();
		// 下载网页html代码
		String url = pageURL;
		if (!url.startsWith("http://"))
			url = "http://" + url;
		if (url.endsWith("/"))
			url = url.substring(0, url.length() - 1);
		html = getHTML(url);
		Pattern p = Pattern.compile("<title>(.*?)</title>");
		Matcher m = p.matcher(html);
		m.find();
		String title = m.group(1);
		File js_img = new File(savePath + "/" + "resource_files");
		if (!js_img.exists()) {
			js_img.mkdir();
			System.out.println(js_img.getPath() + "文件夹不存在，已创建！");
		}
		
		// 下载js脚本和图片
		ArrayList<String> srcList = new ArrayList<String>();
		String srcReg = "http://([^<>'+()]*?)\\.(js|css|jpeg|jpg|png|gif|bmp)";
		p = Pattern.compile(srcReg);
		m = p.matcher(html);
		StringBuffer sb = new StringBuffer();
		while (m.find()) {
			String src=m.group();
//			System.out.println(m.group());
			srcList.add(src);
			int i = src.indexOf("?");
			if (i != -1)
				src = src.substring(0, i);
			downloadFile2(src,
					js_img + "/" + src.substring(src.lastIndexOf("/") + 1));
			m.appendReplacement(sb, "./resource_files/"+src.substring(src.lastIndexOf("/")+1));
		}
		m.appendTail(sb);
//		System.out.println(sb.toString());
		System.out.println(srcList.size());
		// 保存网页HTML到文件
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(savePath
					+ "/" + title + ".html"));
			bw.write(sb.toString());
			bw.flush();
			bw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("网页" + pageURL + "下载成功！");
	}

	// 根据指定的URL下载html代码
	public static String getHTML(String pageURL) {
		StringBuffer pageHTML = new StringBuffer();
		try {
			URL url = new URL(pageURL);
			HttpURLConnection connection = (HttpURLConnection) url
					.openConnection();
			BufferedReader br = new BufferedReader(new InputStreamReader(
					connection.getInputStream()));
			// BufferedWriter fileWriter=new BufferedWriter(new
			// FileWriter(path));
			String line = null;
			while ((line = br.readLine()) != null) {
				// fileWriter.write(line);
				pageHTML.append(line);
				pageHTML.append("\r\n");
			}
			connection.disconnect();
			// fileWriter.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return pageHTML.toString();
	}

	/**
	 * 根据URL下载某个文件
	 * 
	 * @param fileURL
	 *            下载地址
	 * @param filePath
	 *            存放的路径
	 */
	public static void downloadFile2(String fileURL, String filePath) {
		File file = new File(filePath);
		if (!file.exists())
			try {
				file.createNewFile();
				StringBuffer sb = new StringBuffer();
				URL url = new URL(fileURL);
				HttpURLConnection connection = (HttpURLConnection) url
						.openConnection();
				byte[] buffer = new byte[1024];
				InputStream is = connection.getInputStream();
				FileOutputStream fos = new FileOutputStream(file);
				int len = 0;
				while ((len = is.read(buffer)) != -1)
					fos.write(buffer, 0, len);
				fos.close();
				is.close();
				connection.disconnect();
			} catch (IOException e) {
				System.out.println("该文件不存在：" + fileURL);
			}

	}

	public static HashSet<String> extactURLS(String mainUrl) {
		HashSet<String> urls = new HashSet<String>();
		String html = getHTML(mainUrl);
		Pattern p = Pattern.compile("href=\"http://([^<>]*?)\\.58\\.com/\"");
		Matcher m = p.matcher(html);
		m.find();
		while (m.find()) {
			urls.add("http://" + m.group(1) + ".58.com");
		}
		System.out.println(urls.size());
		return urls;
	}

	public boolean downloadFile(String fileURL, String filePath) {
		try {
			File file = new File(filePath);
			file.createNewFile();
			StringBuffer sb = new StringBuffer();
			URL url = new URL(fileURL);
			HttpURLConnection connection = (HttpURLConnection) url
					.openConnection();
			BufferedReader br = new BufferedReader(new InputStreamReader(
					connection.getInputStream()));
			PrintWriter fileWriter = new PrintWriter(new FileWriter(filePath));
			String line = null;
			while ((line = br.readLine()) != null) {
				fileWriter.println(line);
			}
			connection.disconnect();
			fileWriter.close();
			return true;
		} catch (IOException e) {
			System.out.println("该文件不存在：" + fileURL);
			return false;
		}

	}

	public static void replaceFile(){
		
	}
}
